Histogram Model: Color Feature

---

Sergei Papulin (papulin.edu@gmail.com)

Contents

• Generating Images
• Defining Color Elements
• Defining Positional Elements
• Creating Histogram
• Querying
• Operations on Histogram Elements
• Data Analysis
• Image Retrieval
• References

Creating virtual environment

This is an optional step. You can skip it and install packages to your current environment.

python -m venv .venv/histtest
source .venv/histtest/bin/activate
pip install \
    numpy==1.19.5 \
    plotly==5.5.0 \
    jupyter==1.0.0 \
    pillow==5.4.1 \
    scikit-image==0.14.2 \
    himpy=0.0.1

Load packages

import sys
from plotly.subplots import make_subplots
import plotly.graph_objects as go

from himpy.histogram import operations
from himpy.executor import Parser, Evaluator
from himpy.utils import E

%load_ext autoreload
%autoreload 2

# dataset generator
from utils.datasets import ColorImageGenerator

# feature extraction
from utils.feature_extraction import (
    FeatureMerger,
    ColorSetTransformer,
    create_histogram,
    create_histogram_
)

# search engine
from utils.search_engine import SearchEngine

# plot
from utils.plot.plotly_plot import (
    show_color_elements,
    show_complete_histogram,
    show_rank_images
)

# image colors
from utils.feature_extraction.color import COLOR_ELEMENTS, COLOR_ELEMENTS_RGB

Generating Images

image_generator = ColorImageGenerator()

image = image_generator.generate(shape=(100, 100), 
                                 steps=(20, 20), 
                                 random_state=1234)

fig = make_subplots(rows=1, cols=1, subplot_titles=("Initial Image [RGB]",))

fig.add_trace(go.Image(z=image, hoverinfo="skip"), row=1, col=1)
fig.update_yaxes(showticklabels=False)
fig.update_xaxes(showticklabels=False)
fig.update_layout(margin=dict(l=20, r=20, t=40, b=20), width=250, height=200)
fig.show()

Defining Color Elements

Low-Level Elements

Stucture of color elements:

[
    {
        "id": id,
        "h": [min, max],
        "s": [min, max],
        "b": [min, max]
    }, ...
]

Parameters:

• id: identification of an element
• h: hue, interval. If you define interval from 230 to 10, it will be divided into two ones: from 230 to 240 and from 0 to 10
• s: saturation, interval
• b: brightness, interval

# Show the first five low-level color elements
print("Total number of color elements:", len(COLOR_ELEMENTS))
COLOR_ELEMENTS[:5]

Total number of color elements: 40

({'id': 'e1', 'h': [40, 54], 's': [30, 79], 'b': [20, 59]},
 {'id': 'e2', 'h': [40, 54], 's': [80, 159], 'b': [20, 59]},
 {'id': 'e3', 'h': [40, 54], 's': [160, 240], 'b': [20, 59]},
 {'id': 'e4', 'h': [55, 104], 's': [0, 29], 'b': [60, 159]},
 {'id': 'e5', 'h': [55, 104], 's': [30, 79], 'b': [20, 59]})

# Display all low-level color elements
show_color_elements(title="Low-level elements (Color Universal set - Uc)")

# Display all low-level color elements
show_color_elements(element_ids={"e18"}, title="e18")

color_transformer = ColorSetTransformer()
color_image = color_transformer.fit_transform(X=image, y=None)
color_image

array([['e29', 'e29', 'e29', ..., 'e38', 'e38', 'e38'],
       ['e29', 'e29', 'e29', ..., 'e38', 'e38', 'e38'],
       ['e29', 'e29', 'e29', ..., 'e38', 'e38', 'e38'],
       ...,
       ['e12', 'e12', 'e12', ..., 'e32', 'e32', 'e32'],
       ['e12', 'e12', 'e12', ..., 'e32', 'e32', 'e32'],
       ['e12', 'e12', 'e12', ..., 'e32', 'e32', 'e32']], dtype='<U10')

color_image_ = color_transformer.transform_to_int(color_image)


fig = make_subplots(rows=1, cols=2, subplot_titles=("Initial Image [RGB]", "Transformed Image [low-level]"))

common_heatmap_args = {
    "zmax": 40,
    "zauto": False,
    "colorscale": "gray",
    "showscale": False,
    "hoverinfo": "skip"
}

fig.add_trace(go.Image(z=image, hoverinfo="skip"), row=1, col=1)
fig.add_trace(go.Heatmap(z=color_image_, **common_heatmap_args), row=1, col=2)
fig.update_yaxes(autorange="reversed", constrain="domain", scaleanchor="x", row=1, col=2)
fig.update_xaxes(constrain="domain", scaleanchor="y", row=1, col=2)
fig.update_yaxes(showticklabels=False)
fig.update_xaxes(showticklabels=False)
fig.update_layout(margin=dict(l=20, r=20, t=40, b=20), width=500, height=200)
fig.show()

High-Level Elements

$$E^G=\left\{E\vert E\subseteq U\right\}$$

parser = Parser()

# Definition of high-level positional elements

Ec_green        = E("e1+e2+e3+e4+e5+e6+e7+e8+e9+e10+e11+e12+e13+e14+e15+e16+e17+e18+e19+e20")
Ec_yellow_green = E("e2+e3+e21+e22+e23+e24+e25+e26+e27+e28+e29+e30")
Ec_red          = E("e31+e32+e33+e34+e35+e36+e37+e38+e39+e40")
Ec_rose         = E("e32+e35+e36+e39+e40")


Ecs = [
    ("green", Ec_green),
    ("yellow_green", Ec_yellow_green),
    ("red", Ec_red),
    ("rose", Ec_rose)
]


# Sets of hight-level color elements (they will be used for the Evaluator below)

Ecs_set = { name: parser.parse_set(Ec.value) for name, Ec in Ecs}
Ecs_set["green"]

{'e1',
 'e10',
 'e11',
 'e12',
 'e13',
 'e14',
 'e15',
 'e16',
 'e17',
 'e18',
 'e19',
 'e2',
 'e20',
 'e3',
 'e4',
 'e5',
 'e6',
 'e7',
 'e8',
 'e9'}

# Display all low-level color elements
for title, Ec_set in Ecs_set.items():
    show_color_elements(element_ids=Ec_set, title=title)

color_filtered_image = color_transformer.filter_elements(color_image, Ecs_set["green"])
color_data_filtered_image = color_transformer.filter_data(color_image, image, Ecs_set["green"])

# Note: As colors have string id, we convert them into integers to plot
color_filtered_image_ = color_transformer.transform_to_int(color_filtered_image)

fig = make_subplots(rows=1, cols=4, 
                    subplot_titles=(
                        "Initial Image [RGB]", 
                        "Transformed Image [low-level]", 
                        "Element Filter [green]", 
                        "Data Filter [green]"
                    ))

common_heatmap_args = {
    "zmax": 40,
    "zauto": False,
    "colorscale": "gray",
    "showscale": False,
    "hoverinfo": "skip"
}

fig.add_trace(go.Image(z=image, hoverinfo="skip"), row=1, col=1)
fig.add_trace(go.Heatmap(z=color_image_, **common_heatmap_args), row=1, col=2)
fig.add_trace(go.Heatmap(z=color_filtered_image_, **common_heatmap_args), row=1, col=3)
fig.add_trace(go.Image(z=color_data_filtered_image, hoverinfo="skip"), row=1, col=4)
fig.update_yaxes(autorange="reversed", constrain="domain", scaleanchor="x", row=1, col=2)
fig.update_xaxes(constrain="domain", scaleanchor="y", row=1, col=2)
fig.update_yaxes(autorange="reversed", constrain="domain", scaleanchor="x", row=1, col=3)
fig.update_xaxes(constrain="domain", scaleanchor="y", row=1, col=3)
fig.update_yaxes(showticklabels=False)
fig.update_xaxes(showticklabels=False)
fig.update_layout(margin=dict(l=20, r=20, t=40, b=20), width=800, height=200)
fig.show()

Creating Histogram

Normalized histogram of an image can be written as

$$H(d)=\left( h(e_1), \cdots, h(e_n) \mid h(x)=h(x \mid d), 0 \leq h(x) \leq 1, \sum_{x\in U}h(x) = 1 \right).$$

Table 1. Basic notations [1]

Symbol	Definition
$$U$$	Universal set (all possible elements that make up data instances)
$$x$$	Element of the universal set, $x \in U$
$$d$$	Data instance
$$N(d)$$	The total number of elements in $d$
$$X$$	High-level element that is a subset of the universal set, $X\subseteq U$
$$H(d)$$	Histogram of a data instance
$$h(x \mid d)$$	Value that corresponds to element $x$ of histogram $H(d)$
$$H(X \mid d)$$	Histogram of high-level element $X$ given a data instance $d$
$$h(x \mid X, d)$$	Value that corresponds to element $x$ of histogram $H(X \mid d)$

# Option 1
hist = create_histogram((color_image,))
hist.to_dict()

{'e1': 0.08,
 'e10': 0.04,
 'e12': 0.04,
 'e16': 0.04,
 'e2': 0.12,
 'e23': 0.08,
 'e29': 0.04,
 'e3': 0.04,
 'e30': 0.04,
 'e31': 0.04,
 'e32': 0.04,
 'e33': 0.04,
 'e38': 0.04,
 'e40': 0.08,
 'e6': 0.12,
 'e7': 0.04,
 'e8': 0.08}

# TODO: add 1-dim case
# Option 2.a
feature_merger = FeatureMerger()
merged_image = feature_merger.fit_transform((color_image,))
merged_image

array([['e29', 'e29', 'e29', ..., 'e38', 'e38', 'e38'],
       ['e29', 'e29', 'e29', ..., 'e38', 'e38', 'e38'],
       ['e29', 'e29', 'e29', ..., 'e38', 'e38', 'e38'],
       ...,
       ['e12', 'e12', 'e12', ..., 'e32', 'e32', 'e32'],
       ['e12', 'e12', 'e12', ..., 'e32', 'e32', 'e32'],
       ['e12', 'e12', 'e12', ..., 'e32', 'e32', 'e32']], dtype='<U10')

# Option 2.b
hist = create_histogram_(merged_image)
hist.to_dict()

{'e1': 0.08,
 'e10': 0.04,
 'e12': 0.04,
 'e16': 0.04,
 'e2': 0.12,
 'e23': 0.08,
 'e29': 0.04,
 'e3': 0.04,
 'e30': 0.04,
 'e31': 0.04,
 'e32': 0.04,
 'e33': 0.04,
 'e38': 0.04,
 'e40': 0.08,
 'e6': 0.12,
 'e7': 0.04,
 'e8': 0.08}

# Plot histogram
hist_elements = sorted(hist.hist_elements().items(), key=lambda x: int(x[0].lstrip("e")))
elements = ["{}".format(el[0]) for el in hist_elements]
values = [el[1].value for el in hist_elements]
colors = ["rgb{}".format(COLOR_ELEMENTS_RGB[el[0]]) for el in hist_elements]


fig = make_subplots(rows=1, cols=2, column_widths=[0.2, 0.8], subplot_titles=("Image", "Histogram"))

fig.add_image(z=image, row=1, col=1, name="image")
fig.add_bar(x=elements, y=values, marker_color=colors, width=0.5, row=1, col=2, name="histogram")

fig.update_xaxes(gridcolor='#bdbdbd', title="Elements", titlefont=dict(color="grey"), row=1, col=2)
fig.update_yaxes(gridcolor='#bdbdbd', title="Counts", titlefont=dict(color="grey"), row=1, col=2)

fig.update_layout(plot_bgcolor='#fefefe', showlegend=False, height=300, width=900, title_text="Initial Data")
fig.show()

show_complete_histogram(hist)

Total number of elements in the image:

len(hist)

17

Non-zero elements of the histogram:

hist.elements()

['e1',
 'e10',
 'e12',
 'e16',
 'e2',
 'e23',
 'e29',
 'e3',
 'e30',
 'e31',
 'e32',
 'e33',
 'e38',
 'e40',
 'e6',
 'e7',
 'e8']

# TODO: total number of histogram elements

Sum of all histogram element values:

hist.sum()

1.0000000000000002

Get a value of the first histogram element:

hist(hist.elements()[0]).sum()

0.08

Get a value of the e1 element:

hist("e1").sum()

0.08

or

hist.hist_elements()["e1"].value

0.08

Querying

If $U$ is a finite set, then the set of all subsets of $U$, or the $\sigma$-algebra over $U$, is defined as follows

$$ E^G=\left\{X\vert X \subseteq U\right\}. $$

Each element $X$ in $E^G$ can be mapped to its histogram $H(X \mid d)$. So, the whole set of histograms for $E^G$ given a data instance $d$ is

$${H}^{G}(d) = \left \lbrace H(X|d) \mid X \in E^G \right \rbrace.$$

And a single mapping $X$ to $H(X \mid x)$ can be defined as follows

$$ H(X \mid d) = \left(h(e_1),\cdots,h(e_n) \mid h(x)=\left\{\begin{array}{l}h(x \mid d)\;\text{if}\;x \in X,\\0\;\text{otherwise}\end{array}\right. \right) \in H^G(d).$$

high_level_elements = Ecs_set

# Initialize an evaluator
evaluator = Evaluator(operations, hist, high_level_elements=high_level_elements)

E1 = E("green")
E2 = E("yellow_green")

E1_expr = parser.parse_string(E1.value)
HE1 = evaluator.eval(E1_expr)

print("Expression for E1:\n{}".format(E1.value))
print("\nThe parsed expressino for E1 in the postfix notation:\n{}".format(E1_expr))
print("\nHistogram of E1 given the image:\n{}".format(HE1.to_dict()))

Expression for E1:
(green)

The parsed expressino for E1 in the postfix notation:
['green']

Histogram of E1 given the image:
{'e10': 0.04, 'e16': 0.04, 'e3': 0.04, 'e6': 0.12, 'e1': 0.08, 'e8': 0.08, 'e12': 0.04, 'e2': 0.12, 'e7': 0.04}

E2_expr = parser.parse_string(E2.value)
HE2 = evaluator.eval(E2_expr)

print("Expression for E2:\n{}".format(E2.value))
print("\nThe parsed expressino for E2 in the postfix notation:\n{}".format(E2_expr))
print("\nHistogram of E2 given the image:\n{}".format(HE2.to_dict()))

Expression for E2:
(yellow_green)

The parsed expressino for E2 in the postfix notation:
['yellow_green']

Histogram of E2 given the image:
{'e30': 0.04, 'e3': 0.04, 'e23': 0.08, 'e2': 0.12, 'e29': 0.04}

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE2, E2.value)

for name in Ecs_set.keys():
    E_expr = parser.parse_string(E(name).value)
    HE = evaluator.eval(E_expr)
    show_complete_histogram(HE, name)

Operations on Histogram Elements

The histogram algebraic system can be denoted as follows

$$\mathcal{H} = \left\langle H^G,\cap,\cup,\land,\dot\lor,\lor,\dot\backslash,\setminus, -, \mathbf{0},\mathbf{1}\right\rangle,$$

where constants are defined as

$$\mathbf{0}=H^0=\left( h(e_1), \cdots, h(e_n) \mid h(x) = 0, x \in U \right)$$$$\mathbf{1}=H^1=H(d)=\left( h(e_1), \cdots, h(e_n) \mid h(x) = h(x \mid d), x \in U, \sum_{x\in U}h(x)=1 \right)$$

Set Operations

UNION

$$H_1 \cup H_2 =\left(h(e_1),\cdots,h(e_n) \mid h(x)=\max(h(x \mid X_1,d), h(x \mid X_2, d)) \right)$$

E_union = E1 + E2
E_union_expr = parser.parse_string(E_union.value)
HE_union = evaluator.eval(E_union_expr)

print("Expression for E_union:\n{}".format(E_union))
print("\nThe parsed expression for E_union in the postfix notation:\n{}".format(E_union_expr))
print("\nHistogram of E_union given the image:\n{}".format(HE_union.to_dict()))
print("\nValue of presence for E_union:\n{}".format(HE_union.sum()))

Expression for E_union:
((green)+(yellow_green))

The parsed expression for E_union in the postfix notation:
['green', 'yellow_green', '+']

Histogram of E_union given the image:
{'e10': 0.04, 'e30': 0.04, 'e16': 0.04, 'e3': 0.04, 'e6': 0.12, 'e1': 0.08, 'e8': 0.08, 'e23': 0.08, 'e12': 0.04, 'e2': 0.12, 'e7': 0.04, 'e29': 0.04}

Value of presence for E_union:
0.7600000000000001

Show the histogram of E_union given the image:

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE2, E2.value)
show_complete_histogram(HE_union, E_union.value)

INTERSECTION

$$ H_1 \cap H_2 =\left(h(e_1),\cdots,h(e_n) \mid h(x)=\min(h(x \mid X_1, d), h(x \mid X_2, d)) \right)$$

Case 1

E_intersect = E1 * E2  # or E1.Intersection(E2)
E_intersect_expr = parser.parse_string(E_intersect.value)
HE_intersect = evaluator.eval(E_intersect_expr)

print("Expression for E_intersect:\n{}".format(E_intersect))
print("\nThe parsed expression for E_intersect in the postfix notation:\n{}".format(E_intersect_expr))
print("\nHistogram of E_intersect given the image:\n{}".format(HE_intersect.to_dict()))
print("\nValue of presence for E_intersect:\n{}".format(HE_intersect.sum()))

Expression for E_intersect:
((green)*(yellow_green))

The parsed expression for E_intersect in the postfix notation:
['green', 'yellow_green', '*']

Histogram of E_intersect given the image:
{'e3': 0.04, 'e2': 0.12}

Value of presence for E_intersect:
0.16

Show the histogram of E_intercept given the image:

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE2, E2.value)
show_complete_histogram(HE_intersect, E_intersect.value)

Case 2

E3 = E("red")
E3_expr = parser.parse_string(E3.value)
HE3 = evaluator.eval(E3_expr)

E_intersect = E1 * E3  # or E1.Intersection(E2)
E_intersect_expr = parser.parse_string(E_intersect.value)
HE_intersect = evaluator.eval(E_intersect_expr)

print("Expression for E_intersect:\n{}".format(E_intersect))
print("\nThe parsed expression for E_intersect in the postfix notation:\n{}".format(E_intersect_expr))
print("\nHistogram of E_intersect given the image:\n{}".format(HE_intersect.to_dict()))
print("\nValue of presence for E_intersect:\n{}".format(HE_intersect.sum()))

Expression for E_intersect:
((green)*(red))

The parsed expression for E_intersect in the postfix notation:
['green', 'red', '*']

Histogram of E_intersect given the image:
{}

Value of presence for E_intersect:
0

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE3, E3.value)
show_complete_histogram(HE_intersect, E_intersect.value)

SUBSTRACTION or EXCEPTION

$$ H_1 \setminus H_2 =\left(h(e_1),\cdots,h(e_n) \mid h(x)=\left\{\begin{array}{l}h(x \mid X_1, d)\;\text{if}\;h(x \mid X_2, d)=0,\\0\;\text{otherwise}\end{array}\right. \right),$$

where $H_i = H(X_i \mid d)$ and $X_i \in E^G$.

Case 1

E_sub = E1 - E2  # or E1.Sub(E2)
E_sub_expr = parser.parse_string(E_sub.value)
HE_sub = evaluator.eval(E_sub_expr)

print("Expression for E_sub:\n{}".format(E_sub))
print("\nThe parsed expression for E_sub in the postfix notation:\n{}".format(E_sub_expr))
print("\nHistogram of E_sub given the image:\n{}".format(HE_sub.to_dict()))
print("\nValue of presence for E_sub:\n{}".format(HE_sub.sum()))

Expression for E_sub:
((green)/(yellow_green))

The parsed expression for E_sub in the postfix notation:
['green', 'yellow_green', '/']

Histogram of E_sub given the image:
{'e10': 0.04, 'e16': 0.04, 'e6': 0.12, 'e1': 0.08, 'e8': 0.08, 'e12': 0.04, 'e7': 0.04}

Value of presence for E_sub:
0.44

Show the histogram of E_sub given the image:

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE2, E2.value)
show_complete_histogram(HE_sub, E_sub.value)

Case 2

E_sub = E1 - E3  # or E1.Sub(E3)
E_sub_expr = parser.parse_string(E_sub.value)
HE_sub = evaluator.eval(E_sub_expr)

print("Expression for E_sub:\n{}".format(E_sub))
print("\nThe parsed expression for E_sub in the postfix notation:\n{}".format(E_sub_expr))
print("\nHistogram of E_sub given the image:\n{}".format(HE_sub.to_dict()))
print("\nValue of presence for E_sub:\n{}".format(HE_sub.sum()))

Expression for E_sub:
((green)/(red))

The parsed expression for E_sub in the postfix notation:
['green', 'red', '/']

Histogram of E_sub given the image:
{'e10': 0.04, 'e16': 0.04, 'e3': 0.04, 'e6': 0.12, 'e1': 0.08, 'e8': 0.08, 'e12': 0.04, 'e2': 0.12, 'e7': 0.04}

Value of presence for E_sub:
0.6000000000000001

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE3, E3.value)
show_complete_histogram(HE_sub, E_sub.value)

Logical Operations

AND

$$H_1\land H_2=\left\{\begin{array}{l}H_1\;\text{if}\;{}^\Sigma H_1 < {}^\Sigma H_2,\\H_2\;\text{otherwise}\end{array}\right.,$$

where

$$ {}^\Sigma H_i(d) = \sum_{x\in X_i}h(x \mid d)$$

E_and = E1 & E2  # or E1.And(E2)
E_and_expr = parser.parse_string(E_and.value)
HE_and = evaluator.eval(E_and_expr)

print("Expression for E_and:\n{}".format(E_and))
print("\nThe parsed expression for E_and in the postfix notation:\n{}".format(E_and_expr))
print("\nHistogram of E_and given the image:\n{}".format(HE_and.to_dict()))
print("\nValue of presence for E_and:\n{}".format(HE_and.sum()))

Expression for E_and:
((green)&(yellow_green))

The parsed expression for E_and in the postfix notation:
['green', 'yellow_green', '&']

Histogram of E_and given the image:
{'e30': 0.04, 'e3': 0.04, 'e23': 0.08, 'e2': 0.12, 'e29': 0.04}

Value of presence for E_and:
0.32

Show the histogram of E_and given the image:

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE2, E2.value)
show_complete_histogram(HE_and, E_and.value)

OR

$$ H_1 \lor H_2 =\left(H(e_1),\cdots,H(e_n) \mid h(x)=\max(h(x \mid X_1, d), h(x \mid X_2, d)) \right)$$

E_or = E1 | E2  # or E1.Or(E2)
E_or_expr = parser.parse_string(E_or.value)
HE_or = evaluator.eval(E_or_expr)

print("Expression for E_or:\n{}".format(E_or))
print("\nThe parsed expression for E_or in the postfix notation:\n{}".format(E_or_expr))
print("\nHistogram of E_or given the image:\n{}".format(HE_or.to_dict()))
print("\nValue of presence for E_or:\n{}".format(HE_or.sum()))

Expression for E_or:
((green)|(yellow_green))

The parsed expression for E_or in the postfix notation:
['green', 'yellow_green', '|']

Histogram of E_or given the image:
{'e10': 0.04, 'e30': 0.04, 'e16': 0.04, 'e3': 0.04, 'e6': 0.12, 'e1': 0.08, 'e8': 0.08, 'e23': 0.08, 'e12': 0.04, 'e2': 0.12, 'e7': 0.04, 'e29': 0.04}

Value of presence for E_or:
0.7600000000000001

Show the histogram of E_or given the image:

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE2, E2.value)
show_complete_histogram(HE_or, E_or.value)

XOR

$$ H_1\dot\lor H_2=\left\{\begin{array}{l}H_1\;\text{if}\;{}^\Sigma H_1 > {}^\Sigma H_2,\\H_2\;\text{otherwise}\end{array}\right..$$

E_xor = E1 ^ E2  # or E1.Xor(E2)
E_xor_expr = parser.parse_string(E_xor.value)
HE_xor = evaluator.eval(E_xor_expr)

print("Expression for E_xor:\n{}".format(E_xor))
print("\nThe parsed expression for E_xor in the postfix notation:\n{}".format(E_xor_expr))
print("\nHistogram of E_xor given the image:\n{}".format(HE_xor.to_dict()))
print("\nValue of presence for E_xor:\n{}".format(HE_xor.sum()))

Expression for E_xor:
((green)#|(yellow_green))

The parsed expression for E_xor in the postfix notation:
['green', 'yellow_green', '#|']

Histogram of E_xor given the image:
{'e10': 0.04, 'e16': 0.04, 'e3': 0.04, 'e6': 0.12, 'e1': 0.08, 'e8': 0.08, 'e12': 0.04, 'e2': 0.12, 'e7': 0.04}

Value of presence for E_xor:
0.6000000000000001

Show the histogram of E_xor given the image:

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE2, E2.value)
show_complete_histogram(HE_xor, E_xor.value)

XSUBSTRACTION

$$ H_1\dot\backslash H_2=\left\{\begin{array}{l}0\;\text{if}\;{}^\Sigma H_2 > 0,\\H_1\;\text{otherwise}\end{array}\right.$$

Case 1

E_xsub = E1.Xsub(E2)
E_xsub_expr = parser.parse_string(E_xsub.value)
HE_xsub = evaluator.eval(E_xsub_expr)

print("Expression for E_xsub:\n{}".format(E_xsub))
print("\nThe parsed expression for E_xsub in the postfix notation:\n{}".format(E_xsub_expr))
print("\nHistogram of E_xsub given the image:\n{}".format(HE_xsub.to_dict()))
print("\nValue of presence for E_xsub:\n{}".format(HE_xsub.sum()))

Expression for E_xsub:
((green)#/(yellow_green))

The parsed expression for E_xsub in the postfix notation:
['green', 'yellow_green', '#/']

Histogram of E_xsub given the image:
{}

Value of presence for E_xsub:
0

Show the histogram of E_xsub given the image:

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE2, E2.value)
show_complete_histogram(HE_xsub, E_xsub.value)

Case 2

E4 = E("e17+e18+e19")  # elements that are not in image
E4_expr = parser.parse_string(E4.value)
HE4 = evaluator.eval(E4_expr)

E_xsub = E1.Xsub(E4)
E_xsub_expr = parser.parse_string(E_xsub.value)
HE_xsub = evaluator.eval(E_xsub_expr)

print("Expression for E_xsub:\n{}".format(E_xsub))
print("\nThe parsed expression for E_xsub in the postfix notation:\n{}".format(E_xsub_expr))
print("\nHistogram of E_xsub given the image:\n{}".format(HE_xsub.to_dict()))
print("\nValue of presence for E_xsub:\n{}".format(HE_xsub.sum()))

Expression for E_xsub:
((green)#/(e17+e18+e19))

The parsed expression for E_xsub in the postfix notation:
['green', 'e17', 'e18', '+', 'e19', '+', '#/']

Histogram of E_xsub given the image:
{'e10': 0.04, 'e16': 0.04, 'e3': 0.04, 'e6': 0.12, 'e1': 0.08, 'e8': 0.08, 'e12': 0.04, 'e2': 0.12, 'e7': 0.04}

Value of presence for E_xsub:
0.6000000000000001

Show the histogram of E_xsub given the image:

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE4, E4.value)
show_complete_histogram(HE_xsub, E_xsub.value)

Data Analysis

Set a query:

query = (E1 + E2) & E3  # (green or yellow-green) and red

Compute a histogram of the query given the image:

expr = parser.parse_string(query.value)
HE = evaluator.eval(expr)

print("\nHistogram of guery given the image:\n{}".format(HE.to_dict()))
print("\nValue of presence:\n{}".format(HE.sum()))

Histogram of guery given the image:
{'e40': 0.08, 'e38': 0.04, 'e32': 0.04, 'e33': 0.04, 'e31': 0.04}

Value of presence:
0.24000000000000002

E_union = E1 + E2
expr = parser.parse_string(E_union.value)
HE_union = evaluator.eval(expr)

show_complete_histogram(HE1, E1.value)
show_complete_histogram(HE2, E2.value)
show_complete_histogram(HE3, E3.value)
show_complete_histogram(HE_union, E_union.value)
show_complete_histogram(HE, query.value)

Image Retrieval

Expression as query

# Images with normal distrubited some elements 
images = [
    image_generator.generate(
        shape=(100, 100), 
        steps=(20, 20), 
        random_state=i+100) 
    for i in range(100)
]

# Create histograms for the images
hists = list()
limit = len(images)

for indx, image in enumerate(images):
    color_image = color_transformer.fit_transform(X=image, y=None)
    hist = create_histogram((color_image,))
    hists.append((indx, hist))
    print("\rCurrent image index: {}/{}".format(indx + 1, limit), end="")

Current image index: 100/100

# Initialize a search engine
search_engine = SearchEngine(hists, parser, evaluator)

TOP_N = 20

# Elements
E1 = E("green")
E2 = E("yellow_green")
E3 = E("red")

# Define your query
query = (E1 + E2) & E3

# Retrieve images using the query
ranked_images = search_engine.retrieve(query, topN=TOP_N)
print("Total retrieved images:", len(ranked_images))
ranked_images[:5]

Total retrieved images: 20

[(61, 0.48000000000000004),
 (98, 0.44000000000000006),
 (2, 0.44),
 (39, 0.44),
 (55, 0.44)]

fig = show_rank_images(images, ranked_images, limit=TOP_N, 
                       title="Top {}: <b>{}</b>".format(TOP_N, query.value))
fig.show()

# TODO: Show the least similar image

Image sample as query

# Generate a new sample image
sample_image = image_generator.generate(
    shape=(100, 100), 
    steps=(20, 20), 
    random_state=1) 

fig = make_subplots(rows=1, cols=1, subplot_titles=("Sample Image",))

fig.add_trace(go.Image(z=sample_image, hoverinfo="skip"), row=1, col=1)
fig.update_yaxes(showticklabels=False)
fig.update_xaxes(showticklabels=False)
fig.update_layout(margin=dict(l=20, r=20, t=40, b=20), width=200, height=200)
fig.show()

# Transform the image to histogram
color_image = color_transformer.transform(X=sample_image)
sample_hist = create_histogram((color_image,))

# Retrieve images similar to the sample
ranked_images__sample = search_engine.retrieve(sample_hist, topN=TOP_N)
print("Total retrieved images:", len(ranked_images__sample))
ranked_images__sample[:5]

Total retrieved images: 20

[(34, 0.6400000000000001),
 (16, 0.5599999999999999),
 (76, 0.5599999999999999),
 (57, 0.52),
 (43, 0.5199999999999999)]

fig = make_subplots(rows=1, cols=1, subplot_titles=("Sample Image",))

fig.add_trace(go.Image(z=sample_image, hoverinfo="skip"), row=1, col=1)
fig.update_yaxes(showticklabels=False)
fig.update_xaxes(showticklabels=False)
fig.update_layout(margin=dict(l=20, r=20, t=40, b=20), width=200, height=200)
fig.show()

fig = show_rank_images(images, ranked_images__sample, 
                       limit=TOP_N, title="Top {}: <b>Sample Image</b>".format(TOP_N))
fig.show()

# TODO: Show the least similar image

References

• Papulin S. Introduction to Histogram Model
• Papulin S. Multidimensional Histogram Model